CD Actual 22/PC Actual CD 22.iso/SHARE/wnt/lemmy30.exe/TCLSCAN.C

C/C++ Source or Header | 1997-07-24 | 26.4 KB | 992 lines

//////////////////////////////////////////////////////////////////////////
//
//  tclscan.c
//
//  Copyright 1997 by James Iuliano & Phil Mercurio
//
//  Permission granted to use this program for development
//  of Lemmy VI editor syntax highlight plug-ins.  Permission
//  for any other use must be obtained by James Iuliano.
//
//    PJM 19970323    First draft, derived from cppscan.c
//    PJM 19970329    Removed preprocessor-line kludge
//    PJM 19970330    Added custom token types
//    PJM 19970330    Added custom keywords (untested)
//    PJM 19970405    Custom keywords working
//    PJM 19970407    First release version
//
//////////////////////////////////////////////////////////////////////////

#include <windows.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <stdlib.h>

#include "syntax.h"

/*
 * Define this macro if using theObjects-4.0
 */
//#define USING_TO

/*
 * Configure for maximum number of known tokens
 */
#define MAX_KNOWN_TOKENS    (1024)

/*
 * Configure for maximum length of a keyword.  Anything
 * longer than this won't even be searched for.
 */
#define MAX_KEYWORD_LEN        (64)

/** No configuration below **/

/*
 * Configure which Lemmy token ID should be returned for each
 * category of text
 */
#define Unknown     (tk_none)
#define TclKeyword     (tk_cust2)
#define TkKeyword     (tk_cust3)
#define CustomKeyword     (tk_cust4)
#define TOKeyword     (tk_cust5)
#define Comment        (tk_comment)
#define Braces        (tk_cust0)
#define Brackets    (tk_cust1)
#define Number        (tk_number)

/*
 * A list of the custom token types.  The order is
 * important, it has to match the usage of tk_cust?
 * above.
 */
static char* CustomTokenTypes[] = {
    "Tcl Braces {}",
    "Tcl Brackets []",
    "Tcl Keyword",
    "Tk Keyword",
    "Custom Keyword",
#ifdef USING_TO
    "tO Keyword",
#endif
};


/*
 * The text of a token along with the type it should be
 * identified as.
 */
typedef struct {
    char*                token;
    parse_token_type    type;
} Token;

/*
 * Everything considered "global" to this parser, including
 * the table of known tokens.
 */
typedef struct {
    Token        tokenTable[MAX_KNOWN_TOKENS];
    int            knownTokens;        // size of tokenTable
    int            flag_period;         // Flag for decimal point on decimal numbers
    int            flag_exp;            // Flag for exponent (E|e) on decimal numbers
} TclScanGlobals;

/*
 * The initial state of the table of known tokens.
 */
static Token InitTokenTable[] = {
#ifdef USING_TO
    {"args",            TOKeyword},
    {"children",            TOKeyword},
    {"clone",            TOKeyword},
    {"defmethod",            TOKeyword},
    {"defobject",            TOKeyword},
    {"defslot",            TOKeyword},
    {"defsuper",            TOKeyword},
    {"descendants",            TOKeyword},
    {"destruct",            TOKeyword},
    {"exists",            TOKeyword},
    {"hasmethod",            TOKeyword},
    {"hasslot",            TOKeyword},
    {"hastrace",            TOKeyword},
    {"hastraceslot",            TOKeyword},
    {"is ",            TOKeyword},
    {"matches",            TOKeyword},
    {"methods",            TOKeyword},
    {"reclaim",            TOKeyword},
    {"reclaimall",            TOKeyword},
    {"slot",            TOKeyword},
    {"slotappend",            TOKeyword},
    {"slotincr",            TOKeyword},
    {"slotindex",            TOKeyword},
    {"slotinsert",            TOKeyword},
    {"slotlength",            TOKeyword},
    {"slotrange",            TOKeyword},
    {"slotreplace",            TOKeyword},
    {"slots",            TOKeyword},
    {"slotsearch",            TOKeyword},
    {"slotsort",            TOKeyword},
    {"slotunique",            TOKeyword},
    {"slotvar",            TOKeyword},
    {"super",            TOKeyword},
    {"trace",            TOKeyword},
    {"traceslot",            TOKeyword},
    {"unslot",            TOKeyword},
    {"untrace",            TOKeyword},
    {"untraceslot",            TOKeyword},
    {"values",            TOKeyword},
    {"vanilla-object",            TOKeyword},
#endif

    {"after",            TclKeyword},
    {"append",            TclKeyword},
    {"array",            TclKeyword},
    {"auto_execok",        TclKeyword},
    {"auto_load",        TclKeyword},
    {"auto_mkindex",    TclKeyword},
    {"auto_reset",        TclKeyword},
    {"bgerror",            TclKeyword},
    {"break",            TclKeyword},
    {"case",            TclKeyword},
    {"catch",            TclKeyword},
    {"cd",                TclKeyword},
    {"clock",            TclKeyword},
    {"close",            TclKeyword},
    {"concat",            TclKeyword},
    {"continue",        TclKeyword},
    {"else",            TclKeyword},
    {"elseif",            TclKeyword},
    {"eof",                TclKeyword},
    {"error",            TclKeyword},
    {"eval",            TclKeyword},
    {"exec",            TclKeyword},
    {"exit",            TclKeyword},
    {"expr",            TclKeyword},
    {"fblocked",        TclKeyword},
    {"fconfigure",        TclKeyword},
    {"file",            TclKeyword},
    {"fileevent",        TclKeyword},
    {"filename",        TclKeyword},
    {"flush",            TclKeyword},
    {"for",                TclKeyword},
    {"foreach",            TclKeyword},
    {"format",            TclKeyword},
    {"gets",            TclKeyword},
    {"glob",            TclKeyword},
    {"global",            TclKeyword},
    {"history",            TclKeyword},
    {"if",                TclKeyword},
    {"incr",            TclKeyword},
    {"info",            TclKeyword},
    {"interp",            TclKeyword},
    {"join",            TclKeyword},
    {"lappend",            TclKeyword},
    {"library",            TclKeyword},
    {"lindex",            TclKeyword},
    {"linsert",            TclKeyword},
    {"list",            TclKeyword},
    {"llength",            TclKeyword},
    {"load",            TclKeyword},
    {"lrange",            TclKeyword},
    {"lreplace",        TclKeyword},
    {"lsearch",            TclKeyword},
    {"lsort",            TclKeyword},
    {"namespace",        TclKeyword},
    {"new",                TclKeyword},
    {"open",            TclKeyword},
    {"otcl_load",        TclKeyword},
    {"otcl_mkindex",    TclKeyword},
    {"package",            TclKeyword},
    {"pid",                TclKeyword},
    {"pkgMkIndex",        TclKeyword},
    {"pkg_mkIndex",        TclKeyword},
    {"proc",            TclKeyword},
    {"puts",            TclKeyword},
    {"pwd",                TclKeyword},
    {"read",            TclKeyword},
    {"regexp",            TclKeyword},
    {"regsub",            TclKeyword},
    {"rename",            TclKeyword},
    {"return",            TclKeyword},
    {"scan",            TclKeyword},
    {"seek",            TclKeyword},
    {"set",                TclKeyword},
    {"socket",            TclKeyword},
    {"source",            TclKeyword},
    {"split",            TclKeyword},
    {"static",            TclKeyword},
    {"string",            TclKeyword},
    {"subst",            TclKeyword},
    {"switch",            TclKeyword},
    {"tcl",                TclKeyword},
    {"tclPkgSetup",        TclKeyword},
    {"tclPkgUnknown",    TclKeyword},
    {"tclvars",            TclKeyword},
    {"tell",            TclKeyword},
    {"time",            TclKeyword},
    {"trace",            TclKeyword},
    {"unknown",            TclKeyword},
    {"unset",            TclKeyword},
    {"unsupported0",    TclKeyword},
    {"update",            TclKeyword},
    {"uplevel",            TclKeyword},
    {"upvar",            TclKeyword},
    {"vwait",            TclKeyword},
    {"while",            TclKeyword},

    {"bell",            TkKeyword},
    {"bind",            TkKeyword},
    {"bindtags",        TkKeyword},
    {"bitmap",            TkKeyword},
    {"button",            TkKeyword},
    {"canvas",            TkKeyword},
    {"checkbutton",        TkKeyword},
    {"clipboard",        TkKeyword},
    {"destroy",            TkKeyword},
    {"dialog",            TkKeyword},
    {"entry",            TkKeyword},
    {"focus",            TkKeyword},
    {"focusNext",        TkKeyword},
    {"frame",            TkKeyword},
    {"grab",            TkKeyword},
    {"grid",            TkKeyword},
    {"image",            TkKeyword},
    {"label",            TkKeyword},
    {"listbox",            TkKeyword},
    {"lower",            TkKeyword},
    {"menu",            TkKeyword},
    {"menubar",            TkKeyword},
    {"menubutton",        TkKeyword},
    {"message",            TkKeyword},
    {"option",            TkKeyword},
    {"optionMenu",        TkKeyword},
    {"options",            TkKeyword},
    {"pack",            TkKeyword},
    {"pack-old",        TkKeyword},
    {"palette",            TkKeyword},
    {"photo",            TkKeyword},
    {"place",            TkKeyword},
    {"popup",            TkKeyword},
    {"radiobutton",        TkKeyword},
    {"raise",            TkKeyword},
    {"scale",            TkKeyword},
    {"scrollbar",        TkKeyword},
    {"selection",        TkKeyword},
    {"send",            TkKeyword},
    {"text",            TkKeyword},
    {"tk",                TkKeyword},
    {"tkButtonDown",    TkKeyword},
    {"tkButtonLeave",    TkKeyword},
    {"tkButtonUp",        TkKeyword},
    {"tkCancelRepeat",    TkKeyword},
    {"tkCheckRadioInvoke",    TkKeyword},
    {"tkEntryBackspace",    TkKeyword},
    {"tkEntryInsert",        TkKeyword},
    {"tkEntrySeeInsert",    TkKeyword},
    {"tkEntrySetCursor",    TkKeyword},
    {"tkEntryTranspose",    TkKeyword},
    {"tkFirstMenu",            TkKeyword},
    {"tkListboxBeginExtend",    TkKeyword},
    {"tkListboxBeginSelect",    TkKeyword},
    {"tkListboxDataExtend",        TkKeyword},
    {"tkListboxUpDown",        TkKeyword},
    {"tkMbMotion",            TkKeyword},
    {"tkMbPost",            TkKeyword},
    {"tkMenuEscape",        TkKeyword},
    {"tkMenuFirstEntry",        TkKeyword},
    {"tkMenuInvoke",        TkKeyword},
    {"tkMenuMotion",        TkKeyword},
    {"tkPostOverPoint",        TkKeyword},
    {"tkSaveGrabInfo",        TkKeyword},
    {"tkScaleActivate",        TkKeyword},
    {"tkScaleButtonDown",        TkKeyword},
    {"tkScaleControlPress",        TkKeyword},
    {"tkScaleIncrement",        TkKeyword},
    {"tkScrollButtonDown",        TkKeyword},
    {"tkScrollButtonUp",        TkKeyword},
    {"tkScrollByPages",        TkKeyword},
    {"tkScrollByUnits",        TkKeyword},
    {"tkTextAutoScan",        TkKeyword},
    {"tkTextPaste",            TkKeyword},
    {"tkTextResetAnchor",        TkKeyword},
    {"tkTextScrollPages",        TkKeyword},
    {"tkTextTranspose",        TkKeyword},
    {"tkTraverseToMenu",        TkKeyword},
    {"tk_textPaste",        TkKeyword},
    {"tkerror",            TkKeyword},
    {"tkvars",            TkKeyword},
    {"tkwait",            TkKeyword},
    {"toplevel",            TkKeyword},
    {"winfo",            TkKeyword},
    {"wm",                TkKeyword}
};



//
//  Private routines (forward referenced)...
//
//     The scan_... routines will typically set the lpPC->ParseState member
//     based on what they find...
//
static int isodigit(char digit);
static int scan_number(char* next_token, parse_context_type *lpPC);

static int tokenCompare(const void* n1, const void* n2);
static parse_token_type idToken(TclScanGlobals* g, char* token);
static void sortTokens(TclScanGlobals* g);
static void addToken(TclScanGlobals* g, char* token, parse_token_type type);


////////////////////////////////////////////////////////////////////////
//
//  API Definitions...
//


// char *ParseEngineName()
//
//     Purpose:
//        Tell Lemmy the name to use internally for this engine.  Up to 32
//        characters.  If two DLLs emit the same name, they will be in conflict
//        and the first one that appears in a directory scan will be used.
//
//     Returns:
//        Null-terminated string with the parse engine name.
//
DllExport(char *) ParseEngineName()
{
   static char name[32];

   strcpy( name, "Tcl/Tk Parser" );
   return(name);
}

// short ParsePropertySheetAvailable()
//
//     Purpose:
//        Tell whether or not this DLL provides a property sheet dialog.
//
//     Returns:
//        0 - The "ParsePropertySheet" function is not supported.
//        1 - The "ParsePropertySheet" function will display a dialog.
//
DllExport(short) ParsePropertiesAvailable()
{
    return(0);
}

// short ParsePropertySheet(HWND hWndParent)
//
//     Purpose:
//        Implement this function to display a property sheet for your
//        parse engine if so desired.  The parent window to reference in
//        your dialog box is passed as a parameter.
//
//     Returns:
//        0 = Successful, -1 = Unsuccessful.
//
DllExport(short) ParseProperties(HWND hWndParent)
{
    return(-1);
}

// short ParseCustomKeywordsSupported()
//
//     Purpose:
//        Tell Lemmy whether or not custom keyword lists are supported by
//        your engine.
//
//     Returns:
//        1 = TRUE  - Lemmy will allocate and initialize the CustomKeyword info
//                    if a keyword list is defined.
//        0 = FALSE - Lemmy will inform the user that custom keyword lists are
//                    not supported.
//
//
DllExport(short) ParseCustomKeywordsSupported()
{
    return(1);
}


// short ParseLoadCustomTokens( parse_context_type *lpPC )
//
//     Purpose:
//        This entry point will be called when Custom token definitions need
//        to be re-initialized from their saved values.  The LoadCustomToken
//        call is provided as a default way to load custom token attributes in
//        the system registry.
//
//     Returns:
//        0 = Successful, -1 = Unsuccessful.
//
//
DllExport(short) ParseLoadCustomTokens( parse_context_type *lpPC )
{
    int i;

    for(i=0; i < lpPC->CustomTokenMax; i++)
        LoadTokenInfo( &(lpPC->CustomToken[i]) );

    return(0);
}

// short ParseSaveCustomTokens( parse_context_type *lpPC )
//
//     Purpose:
//        If Lemmy edits the Custom Token definitions for your parse engine,
//        and the user wants to save the new definitions, this routine will
//        be called.  An implementation of this routine should save the CustomToken
//        data in a way that it can be reloaded when ParseConstructor is called
//        on a future run.  The SaveCustomToken call is provided as a default
//        way to save custom token attributes in the system registry.
//
//     Returns:
//        0 = Successful, -1 = Unsuccessful.
//
//
DllExport(short) ParseSaveCustomTokens( parse_context_type *lpPC )
{
    int i;

    for(i=0; i < lpPC->CustomTokenMax; i++)
        SaveTokenInfo( &(lpPC->CustomToken[i]) );

    return(0);
}



// Return a malloc'd copy of a string
//
static char* copyOf(char* s)
{
    char* t;

    t = malloc(strlen(s)+1);
    if(t == NULL) return(NULL);

    strcpy(t,s);
    return(t);
}


//    Purpose:
//       This is called during initialization of the parse engine(s) by Lemmy.
//      Initialize variables and allocate storage here.
//
//    Returns:
//       0 = Success, -1 = Failure.
//
//       lpPC->lpExtra
//          Optional storage used by your engine can be allocated, and a
//          reference to it placed in this "void" pointer field.
//
DllExport(short) ParseConstructor( parse_context_type *lpPC )
{
    int i, n, nCustom;
    TclScanGlobals* g;


    // Handle the custom token types for Tcl
    //
    nCustom = sizeof(CustomTokenTypes)/sizeof(char*);

    lpPC->CustomToken = (custom_token_type*) malloc(sizeof(custom_token_type) * nCustom);
    if(lpPC->CustomToken == NULL) return(-1);

    lpPC->CustomTokenMax = nCustom;
    for(i=0; i < nCustom; i++)
        if((lpPC->CustomToken[i].label = copyOf(CustomTokenTypes[i])) == NULL) return(-1);

    ParseLoadCustomTokens(lpPC);

    // Allocate and initialize the TclScanGlobals
    //
    lpPC->lpExtra =  malloc(sizeof(TclScanGlobals));
    if(lpPC->lpExtra == NULL) return(-1);

    g = (TclScanGlobals*) lpPC->lpExtra;

    n = sizeof(InitTokenTable)/sizeof(Token);

    for(i=0; i < n; i++) {
        g->tokenTable[i].token = InitTokenTable[i].token;
        g->tokenTable[i].type = InitTokenTable[i].type;
    }

    g->knownTokens = n;

    // Add the custom keywords
    //
    if(lpPC->CustomKeywordList) {
        for(i=0; i < lpPC->CustomKeywordMax; i++)
            addToken(g,lpPC->CustomKeywordList[i].value,CustomKeyword);
    }

    // Sort the keyword table to make it bsearch()able
    //
    sortTokens(g);

    g->flag_period = FALSE;
    g->flag_exp    = FALSE;

    return(0);
}



// short ParseDestructor( parse_context_type *lpPC )
//
//     Purpose:
//        Called when parse engine objects are being destroyed.  Any memory
//        allocated in ParseConstructor should be freed here.
//
//     Returns:
//        0 = Success, -1 = Failure.
//
//        lpPC->lpExtra
//           If memory referenced here was allocated in the constructor, free
//           it here.  Lemmy is not responsible for the content of this pointer.
//
DllExport(short) ParseDestructor( parse_context_type *lpPC )
{
    int i;

    for(i=0; i < lpPC->CustomTokenMax; i++)
        free(lpPC->CustomToken[i].label);

    free(lpPC->CustomToken);
    free(lpPC->lpExtra);
    return(0);
}


// short ParseInitRedraw( parse_context_type *lpPC )
//
//     Purpose:
//        Called at the beginning of a file redraw.  Make any initializations
//        here on your optional storage.
//
//     Returns:
//        0 = Success, -1 = Failure.
//
DllExport(short) ParseInitRedraw( parse_context_type *lpPC )
{
   return(0);
}



// short ParseInitParser( parse_context_type *lpPC )
//
//     Purpose:
//        Called when the parse state should be reset (can be called more than
//        once during a redraw).  Note:  It is not necessary to call this routine
//        from ParseInitRedraw - it will be called as well automatically.
//
//     Returns:
//        0 = Success, -1 = Failure.
//
DllExport(short) ParseInitParser( parse_context_type *lpPC )
{
   return(0);
}



// short ParseBackscan( parse_context_type *lpPC )
//
//     Purpose:
//        Called during "backscan" initialization.  Lemmy steps backward through
//        the file looking for a pattern on a line that indicates a good place
//        to start parsing from (i.e.: the beginning of a multi-line comment
//        block).  There is a settable "backscan" limit in Lemmy that will limit
//        the number of times this routine is called during initialization.
//
//     Parameters:
//        lpPC->lpParseText
//           The text to be scanned.  Do not update the pointer, or change the
//           text.
//
//     Returns:
//        0 = Success, -1 = Failure. (there should rarely be reason for this
//           call to fail).
//
//        lpPC->FinishBackscan
//           Set to 1 (true) when you have found a pattern that you want to stop
//           "backscanning" on.
//
DllExport(short) ParseBackscan( parse_context_type *lpPC )
{
   char *text;

   text = lpPC->lpParseText;

   if ( (strlen( text ) >= 2) &&
        ( text[strlen(text)-1] == '\\') )
   {
      return(0);
   }

   lpPC->FinishBackscan = TRUE;
   return(0);
}






// short ParseScanPreprocessor( parse_context_type *lpPC )
//
//     Purpose:
//        Test the line to see if its a preprocessor line.   Check the parse
//        state in the event that this is a continuation of a multi-line
//        pre-processor statement.
//
//     Parameters:
//        lpPC->lpParseText (Input/Output)
//           The text to be scanned.  Update this pointer - past the scanned
//           text on return (i.e. pointing at the next token or pointing at
//           the string's null terminator).
//
//           *** In this routine, the pointer should only be moved if the
//           routine determines that this is a preprocessor line.
//
//        lpPC->ParseState (Input/Output)
//           The current parse state.  This is either a fresh start, or a
//           continuation of some sort.  If the value is "st_continue_previous",
//           then look at lpPC->ScannedToken to find out what token we are
//           in the middle of.
//
//           On return, set this (typically) to either st_fresh_start if the
//           end of the token was found, or st_continue_previous if the token
//           continues to the next line.
//
//        lpPC->ScannedToken (Input/Output)
//           Set to the token that was detected.  For this particular routine
//           the valid values would either be "tk_none" or "tk_preprocessor".
//           Any other construct is detected in the ParseToken call.
//
//    TclScan: Not relevant
//
DllExport(short) ParseScanPreprocessor( parse_context_type *lpPC )
{
    return(0);
}



// short ParseToken( parse_context_type *lpPC )
//
//     Purpose:
//        Finally - this is the workhorse.  Scan anything except preprocessor
//        text.
//
//     Parameters:
//        lpPC->lpParseText (Input/Output)
//           The text to be scanned.  Update this pointer - past the scanned
//           text on return (i.e. pointing at the next token or pointing at
//           the string's null terminator).
//
//        lpPC->ParseState (Input/Output)
//           The current parse state.  This is either a fresh start, or a
//           continuation of some sort.  If the value is "st_continue_previous",
//           then look at lpPC->ScannedToken to find out what token we are
//           in the middle of.
//
//           On return, set this (typically) to either st_fresh_start if the
//           end of the token was found, or st_continue_previous if the token
//           continues to the next line.
//
//        lpPC->ScannedToken (Input/Output)
//           On entry, this field indicates the token being scanned from the
//           previous line if lpPC->ParseState is not set to "st_fresh_start.
//
//           On return, set to the token that was detected.
//
DllExport(short) ParseToken( parse_context_type *lpPC )
{
    char  *next_token;
    char  *tmp_pointer;
    char      akeyword[MAX_KEYWORD_LEN];        //
    TclScanGlobals* g = (TclScanGlobals*) lpPC->lpExtra;

    g->flag_period = FALSE;
    g->flag_exp    = FALSE;

    next_token = SkipWhiteSpace( lpPC->lpParseText );

    if(lpPC->ScanColumn == 0) {    // beginning of line
        if(*next_token == '#') {        // comment line
            lpPC->lpParseText = &lpPC->lpParseText[strlen(lpPC->lpParseText)]; // null terminator
            lpPC->ParseState = st_fresh_start;
            lpPC->ScannedToken = Comment;

            return(0);
        }
    }

    //
    //  '.' could be either struct or float - advance over the period if float
    //
    if(*next_token == '.' && isdigit(*(next_token+1))) {
        g->flag_period = TRUE;
        next_token++;
    }

    //
    //  '_' is really considered alpha
    //  If punctuation of some type...figure it out
    //
    if(ispunct( *next_token ) && *next_token != '_') {
        if(*next_token == '[' || *next_token == ']') {
            next_token++;

            lpPC->lpParseText = SkipWhiteSpace(next_token);
            lpPC->ScannedToken = Brackets;
            return(0);
        }
        else if(*next_token == '{' || *next_token == '}') {
            next_token++;

            lpPC->lpParseText = SkipWhiteSpace(next_token);
            lpPC->ScannedToken = Braces;
            return(0);
        }
        // If next token is of COMMENT (;# variety) type
        else if ( *next_token == ';' && *(next_token + 1) == '#' ) {
            lpPC->lpParseText = &lpPC->lpParseText[strlen(lpPC->lpParseText)]; // null terminator
            lpPC->ParseState = st_fresh_start;
            lpPC->ScannedToken = Comment;

            return(0);
        }

        //
        //  Default...
        //
        else {
            ++(lpPC->lpParseText);
            lpPC->ParseState   = st_fresh_start;
            lpPC->ScannedToken = Unknown;
            return(0);
        }
    }

    // Number of some type ... decimal, hex, or octal
    // If scan_number() doesn't recognize it, we still attempt to
    // match it as a keyword.
    //
    else if(isdigit(*next_token) && scan_number(next_token, lpPC) == 0) {
        return(0);
    }

    // Keywords...
    else {
        int tokenLen = 0;
        tmp_pointer = next_token;

        //
        //  Suck it in...
        //
        while (( isalnum( *tmp_pointer ) || *tmp_pointer == '_' ) && *tmp_pointer ) {
            tmp_pointer++;
        }

        tokenLen = tmp_pointer - next_token;
        lpPC->lpParseText = SkipWhiteSpace(tmp_pointer);

        lpPC->ParseState = st_fresh_start;
        if(tokenLen >= MAX_KEYWORD_LEN) {
            lpPC->ScannedToken = Unknown;
        }
        else {
            strncpy( akeyword, next_token, (tmp_pointer - next_token) );
            akeyword[ tmp_pointer - next_token ] = '\0';

            lpPC->ScannedToken = idToken((TclScanGlobals*)lpPC->lpExtra,akeyword);
        }

        return(0);
    }
}



////////////////////////////////////////////////////////////////////////
//
//  Internal Routines...
//
////////////////////////////////////////////////////////////////////////

// Returns TRUE if digit is an octal digit
//
static int isodigit( char digit )
{
    if ( digit >= '0' && digit <= '7' )
        return( TRUE );

    return( FALSE );
}


// Called when the next character is a digit, attempts to consume
// a number according to C's styles.  Code to handle trailing LlUu's etc.
// has been removed, see the original cppscan if you need to recognize
// that kind of number.
//
// If we return 0 (success), ParseToken() should immediately return 0.
// However, if we don't get a perfectly-formatted number here, we don't
// assume that it's Unknown.  If we return -1, ParseToken() should attempt
// to examine this token further.
//
static int scan_number(char* next_token, parse_context_type *lpPC)
{
    TclScanGlobals* g = (TclScanGlobals*) lpPC->lpExtra;

    // Hex
    if ( *next_token == '0' && *(next_token + 1) =='x' && !g->flag_period ) {
        next_token += 2;

        // To catch errors like '0x'... incomplete hex
        if ( !isxdigit( *next_token ))
            return(-1);

        while ( isxdigit( *next_token ) && *next_token )
            next_token++;

        if ( isalpha( *next_token ) )
            return(-1);

        lpPC->lpParseText    = SkipWhiteSpace(next_token);
        lpPC->ParseState   = st_fresh_start;
        lpPC->ScannedToken = Number;
    }

    // Octal..
    else if ( *next_token == '0' && isdigit( *(next_token + 1) ) && !g->flag_period ) {
        next_token += 1;

        while ( isodigit( *next_token ) && *next_token ) {
            next_token++;
        }

        // Trap is here */
        if ( isalnum( *next_token ) )
            return(-1);


        lpPC->lpParseText = SkipWhiteSpace(next_token);
        lpPC->ParseState = st_fresh_start;
        lpPC->ScannedToken = Number;
        return(0);
    }  // end of octal

    /* Must be decimal */
    else {
Decimal:
        while ( isdigit( *next_token ) && *next_token )
            next_token++;

        // Handle decimal place in a number...insure only one
        if ( *next_token == '.' ) {
            next_token++;

            /* Period already set...? strange number :-) */
            if ( g->flag_period )
                return(-1);
            else {
                g->flag_period = TRUE;
                goto Decimal;
            }
        }

        // Handle [Ee] for floats
        if ( tolower( *next_token ) == 'e' ) {
            next_token++;

            // Second [Ee]?
            if ( g->flag_exp )
                return(-1);
            else {
                g->flag_exp = TRUE;

                // Absorb sign if present
                if ( *next_token == '+' || *next_token == '-' ) {
                    next_token++;
                }

                // Make sure next value is a number for exponent
            if ( !isdigit( *next_token ) )
                return(-1);

            // Continue parsing otherwise
            goto Decimal;
            }
        }

        if ( isalpha( *next_token ) )
            return(-1);

        lpPC->lpParseText = SkipWhiteSpace(next_token);
        lpPC->ParseState = st_fresh_start;
        lpPC->ScannedToken = Number;
        return(0);
    }  // end of decimal

    return(0);        // just in case it's missed above
}

// qsort() and bsearch() comparison function for Tokens
//
static int tokenCompare(const void* n1, const void* n2)
{
    return(strcmp(((const Token*)n1)->token, ((const Token*)n2)->token));
}

// Return the parse type for the given token.  If not recognized,
// tk_none is returned.
//
parse_token_type idToken(TclScanGlobals* g,char* token)
{
    Token*    t;
    Token    key;

    key.token = token;
    t = (Token*) bsearch(&key,g->tokenTable,
        g->knownTokens,sizeof(Token),tokenCompare);

    if(t == NULL)
        return(Unknown);
    else
        return(t->type);
}

// Sort the token table
//
static void sortTokens(TclScanGlobals* g)
{
    qsort(g->tokenTable,g->knownTokens, sizeof(Token), tokenCompare);
}

// Add a new token to the table.  Call sortTokens() after 1 or
// more invocations of this function.
//
// If we have no more room, quietly refuse to add the token.
//
static void addToken(TclScanGlobals* g, char* token, parse_token_type type)
{
    if(g->knownTokens >= MAX_KNOWN_TOKENS) return;

    g->tokenTable[g->knownTokens].token = token;
    g->tokenTable[g->knownTokens].type = type;

    g->knownTokens++;
}